library(data.table)
library(dplyr)
library(reshape2)
library(sampling)
library(caret)
library(party)
library(pROC)

setwd('F:/ZHZ_R/LOST/')

#-----1 基于3+N、4+N、5+N 月数据建立分类器，参考查全82%，差准44%-----
fun1<-function(fname1,rs,ns,fname2,mark){
  dt<-fread(fname1,
            sep=',',integer64 = 'numeric')
  dt<-as.data.frame(dt)
  dt<-dt[,rs]
  names(dt)[1:7]<-ns
  dt<-as.data.table(dt)
  dt2<-fread(fname2,
             sep=',',integer64 = 'numeric')
  dt<-dt2[dt,on='mobile']
  dt[is.na(dt)]<-0
  for(i in 2:length(names(dt))){
    names(dt)[i]<-paste(names(dt)[i],mark,sep="")
  }
  return(dt)
}
#导入-2 -1月数据
dt.3.all<-fun1('201805_RFV_全量清单.csv',
               c('mobile','ck',"xia","zhong","shang","rfv_f","vv_sum"),
               c('mobile','changke',"xia","zhong","shang","days","vv_sum"),
               'RFV_V_按pv汇总_201805.csv',3)#

dt.4.all<-fun1('201806_RFV_全量清单.csv',
               c('mobile','ck',"xia","zhong","shang","rfv_f","vv_sum"),
               c('mobile','changke',"xia","zhong","shang","days","vv_sum"),
               'RFV_V_按pv汇总_201806.csv',4)#

dt.4.all<-dt.3.all[dt.4.all,on='mobile']
dt.4.all[is.na(dt.4.all)]<-0
rm(dt.3.all)

dt.4.all<-dt.4.all[,avg_sum3:=vv_sum3/days3]
dt.4.all<-dt.4.all[,avg_sum4:=vv_sum4/days4]
dt.4.all[is.na(dt.4.all)]<-0

test<-dt.4.all[,.(
  mobile,
  busi=业务4-业务3,
  qiandao=签到4-签到3,
  chaxun=查询4-查询3,
  daohang=底部切换4-底部切换3,
  digital=数字化内容4-数字化内容3,
  changke=changke3+changke4,
  shang=shang4-shang3,
  zhong=zhong4-zhong3,
  xia=xia4-xia3,
  days=days4-days3,
  vv_sum=vv_sum4-vv_sum3,
  avg_sum=avg_sum4-avg_sum3
)]

dt.4.all<-test[dt.4.all,on='mobile']
dt.4.all[is.na(dt.4.all)]<-0
rm(test)

#导入0月数据
dt.5<-fread('RFV_F_201807.csv',
            sep=',',integer64 = 'numeric')
dt.5<-dt.5[,.(mobile,lost_5=0)]

dt.4.all<-dt.5[dt.4.all,on='mobile']
dt.4.all[is.na(dt.4.all)]<-1
rm(dt.5)

#建模建立分类器
dt<-dt.4.all
rm(dt.4.all)
cl<-c(names(dt)[!names(dt) %in% 
                  c('mobile','lost_5',"分享3","业务3" ,"签到3" ,"礼包3" , "查询3",
                    "充值3" ,"登录设置等其他服务3","底部切换3","关闭弹框3",
                    "自动3","搜索3","活动3","数字化内容3","qita3","changke3",
                    "xia3","zhong3","shang3","days3","vv_sum3","avg_sum3",
                    "礼包4" ,"登录设置等其他服务4","关闭弹框4","qita4","vv_sum4",
                    "vv_sum","自动4","分享4")])
#抽样
train.id<-strata(dt,stratanames = "lost_5",size = c(50000,50000),description=T)#存留、流失
train.dt<-getdata(dt,train.id)
train.dt<-as.data.frame(train.dt)
train.dt$lost_5<-as.factor(train.dt$lost_5)
train.dt<-train.dt[,c("lost_5",cl)]

#抽取测试集
dt.0.t<-as.data.frame(dt)
test.dt<-dt.0.t[sample(nrow(dt.0.t),10000),c("lost_5",cl)]
test.dt.all<-dt.0.t[,c("lost_5",cl)]
test.dt.input<-test.dt[,cl]
rm(dt.0.t)

#选中参数，开始预测
cl.LR.SELECT<-cl
#cl.LR.SELECT<-c('chaxun','changke','签到4','xia4','days4','avg_sum4')
train.LR.TEMP<-train.dt[,c("lost_5",cl.LR.SELECT)]
test.LR.TEMP<-test.dt.all[,c("lost_5",cl.LR.SELECT)]
# test.LR.TEMP<-as.data.frame(test.LR.TEMP)
train.LR.TEMP$lost_5<-as.double(train.LR.TEMP$lost_5)
model_LR<-glm(lost_5 ~ .,family=binomial(link='logit'),data=train.LR.TEMP)

train.LR.TEMP.aov<-aov(lost_5 ~ busi+qiandao,data=train.LR.TEMP)

summary(train.LR.TEMP.aov)
summary(model_LR
        )
model_LR.step<-step(model_LR,direction = "both")


pre<-predict(model_LR,type='response')
r_c<-roc(train.dt$lost_5, pre, plot=TRUE, print.thres=TRUE, print.auc=TRUE)  

c.pred<-predict(model_LR,test.LR.TEMP,type='response')
c.pred <- ifelse( c.pred > 0.49,1,0)
a<-t(table(c.pred,test.LR.TEMP$lost_5))
LR_recall<-a[2,2]/(a[2,1]+a[2,2])
LR_precision<-a[2,2]/(a[1,2]+a[2,2])
LR_F1<-2*LR_recall*LR_precision/(LR_recall+LR_precision)
LR_F2<-5*LR_recall*LR_precision/(LR_recall+4*LR_precision)

print(a)
print(LR_recall)
print(LR_precision)
print(LR_F1)




library(corrplot)



b<-cor(train.LR.TEMP)

corrplot(b)


a<-princomp(train.LR.TEMP,cor=TRUE)

summary(a)




model_LR$coefficients
library(smbinning)
train.LR.TEMP$lost_5<-as.numeric(train.LR.TEMP$lost_5)
train.LR.TEMP$lost_52<-train.LR.TEMP$lost_5-1

factor_vars <- NULL#'avg_sum',
continuous_vars <- c("changke", "changke4","busi", "qiandao","chaxun", "daohang", "digital",
                     "shang",'zhong','xia','days',
                     '业务4','签到4','查询4','充值4', '底部切换4',
                     '搜索4', '活动4', '数字化内容4', 'xia4',
                     'zhong4', 'shang4','days4', 'avg_sum4')


iv_df <- data.frame(VARS=c(factor_vars, continuous_vars), IV=numeric(25))  # init for IV results

# compute IV for categoricals
for(factor_var in factor_vars){
  smb <- smbinning.factor(train.LR.TEMP, y="lost_52", x=factor_var)  # WOE table
  if(class(smb) != "character"){ # heck if some error occured
    iv_df[iv_df$VARS == factor_var, "IV"] <- smb$iv
  }
}
# compute IV for continuous vars
for(continuous_var in continuous_vars){
  print(continuous_var)
  smb <- smbinning(train.LR.TEMP, y="lost_52", x=continuous_var)  # WOE table
  
  if(class(smb) != "character"){  # any error while calculating scores.
    iv_df[iv_df$VARS == continuous_var, "IV"] <- smb$iv
  }
}

iv_df <- iv_df[order(-iv_df$IV), ]  # sort
iv_df




#相关性
library('psych')
train.LR.TEMP<-train.dt[,c(cl.LR.SELECT)]
names(train.LR.TEMP)[12:19]
setnames(train.LR.TEMP,12:19,c('YeWu4','QianDao4','ChaXun4','Chongzhi4',
                               'DiBu4','SouSuo4','HuoDong4','ShuZiHua4'))
a<-corr.test(train.LR.TEMP)

#数据探索
colSums(is.na(train.LR.TEMP))
train.LR.TEMP<-as.data.table(train.LR.TEMP)
train.LR.TEMP[,.N/nrow(train.LR.TEMP),chaxun]


#------------技能大赛------------------------------------------
dt.update<-fread("F:\\ZHZ_R\\RFVALL\\RFV整合201803-201807.csv"
                 ,integer64 = 'numeric')

setwd("F:\\ZHZ_R\\LOST\\V2\\")

dt.5.pro<-fread("大数据大赛5月掌厅yly_20180808_104914_222698.csv",
                integer64 = 'numeric')
dt.6.pro<-fread("大数据大赛6月掌厅yly_20180808_110714_222706.csv",
                integer64 = 'numeric')

dt.5.pv<-fread("RFV_V_按pv汇总_201805.csv",
               integer64 = 'numeric')
dt.6.pv<-fread("RFV_V_按pv汇总_201806.csv",
               integer64 = 'numeric')

dt.5.rfv<-fread("201805_RFV_全量清单.csv",
              integer64 = 'numeric')
dt.6.rfv<-fread("201806_RFV_全量清单.csv",
                integer64 = 'numeric')

# "分享"   "业务"   "签到"  "礼包"   "查询"   "充值"   "登录设置等其他服务"
# [9] "底部切换"   "关闭弹框"           "自动"   "搜索"  
# [13] "活动"   "数字化内容"         "qita"     
newNames<-c("FX",'YW','QD','LB','CX','CZ','DL','DH','GB','ZD','SS','HD','SZ','QT')
setnames(dt.5.pv,2:15,newNames)
setnames(dt.6.pv,2:15,newNames)
dt.5.pv<-dt.5.pv[,.(mobile,FX,YW,QD,LB,CX,CZ,SS,HD,SZ)]
dt.6.pv<-dt.6.pv[,.(mobile,FX,YW,QD,LB,CX,CZ,SS,HD,SZ)]
pp<-function(dt,dt.up){
  
  setnames(dt,1,'mobile')
  dt.up<-dt.up[,.(mobile,rfv_f,vv_sum,vv_avg,shang,zhong,xia)]
  dt.up<-dt.up[dt.up$mobile %in% dt$mobile]
  dt.up<-dt[dt.up,on='mobile']
  #缺失处理
  dt.up$GRP_TOGETHER_FLAG[is.na(dt.up$GRP_TOGETHER_FLAG)]<-0
  dt.up$CROWD_MAIN_PHONE_NO_FLAG[is.na(dt.up$CROWD_MAIN_PHONE_NO_FLAG)]<-0
  dt.up$DZD_TYPE[is.na(dt.up$DZD_TYPE)]<-0
  for(i in "AGE")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$AGE,na.rm = T))
  for(i in "ONLINE_ID")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$ONLINE_ID,na.rm = T))
  for(i in "GPRS_MISS_FLOW")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$ONLINE_ID,na.rm = T))
  for(i in "BFR_ALCT_TOTAL_FEE")
    set(dt.up,i = which(is.na(dt.up[[i]])),j=i,value = median(dt.up$ONLINE_ID,na.rm = T))
  dt.up<-dt.up[DZD_TYPE==1,DZD_TYPE:=-1]
  dt.up<-dt.up[DZD_TYPE==2,DZD_TYPE:=1]
  
  return(dt.up)
}

dt5<-pp(dt.5.pro,dt.5.rfv)
dt6<-pp(dt.6.pro,dt.6.rfv)

dt5<-dt.5.pv[dt5,on='mobile']
dt5<-dt5[!is.na(dt5$FX)]

dt6<-dt.6.pv[dt6,on='mobile']
dt6<-dt6[!is.na(dt6$FX)]

rm(dt.5.pro,dt.6.pro,
   dt.5.rfv,dt.6.rfv,
   dt.5.pv,dt.6.pv)


for(i in 2:length(names(dt5))){
  names(dt5)[i]<-paste(names(dt5)[i],"_l",sep="")
}


dt.delta<-dt5[dt6,on='mobile']
dt.delta<-dt.delta[!is.na(dt.delta$FX_l)]

dt.delta2<-dt.delta[,.(
  mobile,
  FX_d=FX_l-FX,
  YW_d=YW_l-YW,
  QD_d=QD_l-QD,
  LB_d=LB_l-LB,
  CX_d=CX_l-CX,
  CZ_d=CZ_l-CZ,
  SS_d=SS_l-SS,
  HD_d=HD_l-HD,
  SZ_d=SZ_l-SZ,
  GPRS_MISS_FLOW_d=GPRS_MISS_FLOW_l-GPRS_MISS_FLOW,
  BFR_ALCT_TOTAL_FEE_d=BFR_ALCT_TOTAL_FEE_l-BFR_ALCT_TOTAL_FEE,
  rfv_f_d=rfv_f_l-rfv_f,
  vv_sum_d=vv_sum_l-vv_sum,
  vv_avg_d=vv_avg_l-vv_avg,
  shang_d=shang_l-shang,
  zhong_d=zhong_l-zhong,
  xia_d=xia_l-xia
)]

colSums(is.na(dt6))
dt.6.allsample<-dt.delta2[dt6,on='mobile']
dt.6.allsample[is.na(dt.6.allsample)]<-0

dt.7lost<-dt.update[,.(mobile,lost201807)]
dt.6.allsample<-dt.7lost[dt.6.allsample,on='mobile']
colSums(is.na(dt.6.allsample))
table(dt.6.allsample$lost201807)

rm(dt.7lost,dt.delta,dt.delta2,dt.update,dt5,dt6)

#---样本数据清洗完成---

sum(dt.6.allsample$lost201807)/nrow(dt.6.allsample)

dt<-dt.6.allsample
rm(dt.6.allsample)
#1:4
#抽样
train.id<-strata(dt,stratanames = "lost201807",size = c(1600000,400000),description=T)#存留、流失
train.dt<-getdata(dt,train.id)

#train.dt$lost201807<-as.factor(train.dt$lost201807)
train.dt<-train.dt[,Prob:=NULL]
train.dt<-train.dt[,ID_unit:=NULL]
train.dt<-train.dt[,Stratum:=NULL]


#抽取测试集
test.dt<-dt[!dt$mobile %in% train.dt$mobile]




#RF重要性
library(randomForest)
rf.train<-randomForest(as.factor(lost201807) ~ .,data=train.dt[,c(2:41)],importance = T)
importance(rf.train)

rf.train.imp<-as.data.frame(rf.train$importance)

rf.train.imp$pmt<-row.names(rf.train.imp)
rf.train.imp<-as.data.table(rf.train.imp)


#相关性
library('psych')
library('reshape2')
train.dt.cor<-corr.test(train.dt[,c(2:40)])

a<-as.data.frame(train.dt.cor$r)
a2<-as.data.frame(train.dt.cor$p)

a$pmt<-row.names(a)
a2$pmt<-row.names(a2)
a3<-melt(a,id.vars = c('pmt'),measure.vars = row.names(a))
a3.p<-melt(a2,id.vars = c('pmt'),measure.vars = row.names(a2))
a3<-a3[a3$pmt!=a3$variable,]
a3.p<-a3.p[a3.p$pmt!=a3.p$variable,]
a3<-as.data.table(a3)
a3.p<-as.data.table(a3.p)

train.dt.cor.re<-a3.p[a3,on=c('pmt','variable')]
rm(a,a2,a3,a3.p)
train.dt.cor.re.high<-train.dt.cor.re[i.value>=0.66 & value<=0.05]

#VI
library(smbinning)
factor_vars <- NULL#'avg_sum',
continuous_vars <- names(train.dt[,c(2:40)])
iv_df <- data.frame(VARS=c(factor_vars, continuous_vars), IV=numeric(39))  # init for IV results

# compute IV for categoricals
# for(factor_var in factor_vars){
#   smb <- smbinning.factor(train.LR.TEMP, y="lost_52", x=factor_var)  # WOE table
#   if(class(smb) != "character"){ # heck if some error occured
#     iv_df[iv_df$VARS == factor_var, "IV"] <- smb$iv
#   }
# }
# compute IV for continuous vars
for(continuous_var in continuous_vars){
  print(continuous_var)
  smb <- tryCatch(
    smbinning(train.dt, y="lost201807", x=continuous_var),
    error=function(e){print("出现错误")} 
  )
  # smb <- smbinning(train.dt, y="lost201807", x=continuous_var)  # WOE table
  if(class(smb) != "character"){  # any error while calculating scores.
    iv_df[iv_df$VARS == continuous_var, "IV"] <- smb$iv
  }
}

iv_df <- iv_df[order(-iv_df$IV), ]  # sort
iv_df

iv_df<-as.data.table(iv_df)
setnames(iv_df,1,'pmt')
iv_df<-rf.train.imp[iv_df,on='pmt']
plot(iv_df$MeanDecreaseGini,iv_df$IV)


#高相关对比筛选
p.a<-train.dt.cor.re.high$pmt
p.b<-as.character(train.dt.cor.re.high$variable)

pmt.cor<-unique(c(p.a,p.b))
pmt.UNcor<-names(train.dt[,c(2:40)])[!names(train.dt[,c(2:40)]) %in% pmt.cor]

p.fail<-c()
for (i  in 1:length(p.a)) {
  if(iv_df[pmt==p.a[i],IV] >=iv_df[pmt==p.b[i],IV]){
    p.fail[i]<-p.b[i]
  }else{
    p.fail[i]<-p.a[i]
  }
}
p.fail<-unique(p.fail)
p.win1<-pmt.cor[!pmt.cor %in% p.fail]

p.fail<-c()
for (i  in 1:length(p.a)) {
  if(iv_df[pmt==p.a[i],MeanDecreaseGini] >=iv_df[pmt==p.b[i],MeanDecreaseGini]){
    p.fail[i]<-p.b[i]
  }else{
    p.fail[i]<-p.a[i]
  }
}
p.fail<-unique(p.fail)
p.win2<-pmt.cor[!pmt.cor %in% p.fail]

#人工选择
p.win<-p.win2
p.final<-unique(c(pmt.UNcor,p.win))


#初步选择
# model_LR<-glm(as.factor(lost201807) ~ .,family=binomial(link='logit'),
#               data=train.dt[,c(2:41)])
# summary(model_LR)
# anova(model_LR, test = 'Chisq')
# 
# #QD_d 15221
# p.all<-names(train.dt[,c(2:41)])
# p.select<-p.all[!p.all %in% c('CX_d','xia_d','CX','xia')]

#-----------------------

p.final<-c("FX_d","YW_d","CX_d",
"CZ_d","SS_d","HD_d",
"SZ_d","GPRS_MISS_FLOW_d","BFR_ALCT_TOTAL_FEE_d",    
"vv_avg_d","xia_d","FX", 
"YW","CX","CZ",
"SS",  "ONLINE_ID","AGE",
"CROWD_MAIN_PHONE_NO_FLAG","GRP_TOGETHER_FLAG","DZD_TYPE",
"GPRS_MISS_FLOW","BFR_ALCT_TOTAL_FEE","vv_avg",
"LB_d","vv_sum_d","rfv_f","HD")

model_LR<-glm(as.factor(lost201807) ~ .,family=binomial(link='logit'),
              data=as.data.frame(train.dt)[,c(p.final,'lost201807')])

summary(model_LR)
#anova(model_LR, test = 'Chisq')
p.final2<-p.final[!p.final %in% c('BFR_ALCT_TOTAL_FEE_d','vv_avg_d')]


pre<-predict(model_LR,type='response')
r_c<-roc(train.dt$lost201807, pre, plot=TRUE, print.thres=TRUE, print.auc=TRUE)  

c.pred<-predict(model_LR,test.dt,type='response')
c.pred <- ifelse( c.pred > 0.22,1,0)
a<-t(table(c.pred,test.dt$lost201807))
LR_recall<-a[2,2]/(a[2,1]+a[2,2])
LR_precision<-a[2,2]/(a[1,2]+a[2,2])
LR_F1<-2*LR_recall*LR_precision/(LR_recall+LR_precision)
LR_F2<-5*LR_recall*LR_precision/(LR_recall+4*LR_precision)

print(a)
print(LR_recall)
print(LR_precision)
print(LR_F1)

model_LR$coefficients

rf.train.imp <- rf.train.imp[order(rf.train.imp$MeanDecreaseGini), ]  # sort
tobetest<-rf.train.imp$pmt
tobetest<-tobetest[tobetest %in% p.final]

p.final2<-p.final
para_test_LR<-c()
para_test_LR_recall<-c()
para_test_LR_precision<-c()
para_test_LR_F1score<-c()
para_test_LR_F2score<-c()
x=1
for(i in 1:(length(tobetest)-3)){
  p.final2<-tobetest[-c(1:i)]
  print(i)
  model_LR<-glm(as.factor(lost201807) ~ .,family=binomial(link='logit'),
                data=as.data.frame(train.dt)[,c(p.final2,'lost201807')])
  #预测 all
  c.pred<-predict(model_LR,test.dt,type='response')
  #r_c<-roc(test.dt$lost_5, c.pred, plot=TRUE, print.thres=TRUE, print.auc=TRUE) 
  c.pred <- ifelse( c.pred > 0.22,1,0)
  #print(table(c.pred))
  a<-t(table(c.pred,test.dt$lost201807))
  #print(a)
  para_test_LR[x]<-paste(c(p.final2),collapse = "_")
  #print(para_test_LR[x])
  para_test_LR_recall[x]<-a[2,2]/(a[2,1]+a[2,2])
  print(para_test_LR_recall[x])
  para_test_LR_precision[x]<-a[2,2]/(a[1,2]+a[2,2])
  print(para_test_LR_precision[x])
  para_test_LR_F1score<-(para_test_LR_recall[x]*para_test_LR_precision[x])/(para_test_LR_recall[x]+para_test_LR_precision[x])
  para_test_LR_F2score<-(5*para_test_LR_recall[x]*para_test_LR_precision[x])/(para_test_LR_recall[x]+4*para_test_LR_precision[x])
  x=x+1
}  
testReport.LR<-data.frame(
  zuhe=para_test_LR,
  precision=para_test_LR_precision,
  recall=para_test_LR_recall,
  f1=para_test_LR_F1score,
  f2=para_test_LR_F2score
)

